const got = require('got');
const urllib = require('url');
const vm = require('vm');
const fs = require('fs');

const catalog = `9K`;

const savefolder = __dirname + `/weiqi/${catalog}`;
let meta = {};
try { fs.mkdirSync(savefolder) } catch (e) { }
try
{
	let text = fs.readFileSync(`${savefolder}\\meta.jsjs`, { encoding: 'utf8' });
	meta = JSON.parse(text);
}
catch (e)
{
}

process.on('SIGINT', e =>
{
//	console.log('exit');
//	fs.writeFileSync(`${savefolder}\\meta.jsjs`, JSON.stringify(meta, null, '    '));
//	process.exit();
});
process.on('exit', e =>
{
//	console.log('exit');
//	fs.writeFileSync(`${savefolder}\\meta.jsjs`, JSON.stringify(meta, null, '    '));
});

if (!meta.done) meta.done = {};
if (!meta.pagedone) meta.pagedone = {};
function wait(ms)
{
	return new Promise((resolve) =>
	{
		setTimeout(resolve, ms);
	})
}
function findall(re, text)
{
	let arr = [];
	let array1;
	while ((array1 = re.exec(text)) !== null)
	{
		arr.push(array1);
	}
	return arr;
}

function AnyRegionString(str)
{
	var i = str.indexOf('href');
	var j = str.indexOf('title');
	var url = str.substring(i + 7,j - 3);
	var k = str.indexOf('二手房');
	var m = str.indexOf('</a>');
	var region = str.substring(k + 6,m);
	outvalue = {};
	outvalue.url = url;
	outvalue.regionName = region;
	return outvalue;
}
function AnySmallRegionString(str)
{
	//<a href="/xiaoqu/beicai/">北蔡</a>
	var i = str.indexOf('href');
	var j = str.indexOf('>');
	var url = str.substring(i + 7,j - 2);
	var m = str.indexOf('</a>');
	var region = str.substring(j + 1,m);
	outvalue = {};
	outvalue.url = url;
	outvalue.regionName = region;
	
	return outvalue;
}

async function grabpage(pageurl)
{
	let pagetext = await got(pageurl);
	pagetext = pagetext.body;
	var texts = pagetext.split('\n')
	console.log(texts.length);
//	console.log(texts[1]);
//
	var lists = []
	for(var i = 0 ; i < texts.length;i++)
	{
		if(texts[i].indexOf('data-role="ershoufang"') != -1)
		{

			for(var j = 1 ; j < 30;j++)
			{
				if(texts[i + j].indexOf('</div>') != -1)
				{
					break;
				}
				else
				{
					if(texts[i + j].indexOf('href') != -1)
					{
						if(texts[i+j].indexOf('周边') == -1)
						{
							lists.push(texts[i+j]);
						}
					}
				}
			}
			break;
		};
	}
	RegionList = [];
	for(var i = 0 ; i < lists.length;i++)
	{
//		console.log(lists[i])		;
		RegionList.push(AnyRegionString(lists[i]));
	}
	for(var i = 0 ; i < RegionList.length;i++)
	{
		await GotRegion(RegionList[i]);
	}
	//	console.log(pagetext);
//	let re = new RegExp(`<a href="(\\/${catalog}\\/[0-9]+\\/)">`, 'g');
//	let allmatches = findall(re, pagetext.body);
//	let allurls = allmatches.map(x => urllib.resolve(pageurl, x[1]));
//	for (let url of allurls)
//	{
//		await grab(url);
//	}
//	meta.pagedone[pageurl] = true;
}

async function GotRegion(RegionInfo)
{
	//https://sh.lianjia.com/xiaoqu/yangpu/
	var finallyurl = 'https://sh.lianjia.com/' + RegionInfo.url;
	console.log(finallyurl);
	let pagetext = await got(finallyurl);
	pagetext = pagetext.body;
	var texts = pagetext.split('\n')
	console.log(texts.length);
	smallregionlist = [];
	for(var i = 0 ; i < texts.length;i++)
	{
		if(texts[i].indexOf('data-role="ershoufang"') != -1)
		{
			for(var j = 10 ; j < 30;j++)
			{
				if(texts[i + j].indexOf('<div>') != -1)
				{
					
					for(var m = 1 ; m < 200 ;m++)
					{
						if(texts[i + j + m].indexOf('</div>') != -1)
						{
							break;
						}		
						if(texts[i + j + m].indexOf('href') != -1)
						{
//							console.log(texts[i+j+m]);
							smallregionlist.push(AnySmallRegionString(texts[i+j+m]));								
						}
					}
					break;
				}
			}
			break;
		}
	}
	for(var i = 0 ;i < smallregionlist.length;i++)
	{
		console.log(smallregionlist[i].url);
		console.log(smallregionlist[i].regionName);
	}
	return smallregionlist;
}

var g_Count = 0;
async function grab(url)
{
	if (meta.done[url])
	{
		console.log('done', url)
		return;
	}
	let text = (await got(url)).body;
//	console.log('got', url);
	var texts = text.split('/n')
//	console.log(texts.length);
	for(var i = 0 ; i < texts.length;i++)
	{
//		console.log(texts[i]);
	}
/*	let header = `var g_qq = {`;
	let idx = text.indexOf(header);
	if (idx < 0)
		throw new Error(`Cant find g_qq at url:` + url);
	let js = '{';
	let m = 1;
	for (let i = idx + header.length; i < text.length; ++i)
	{
		js += text[i];
		switch (text[i])
		{
			case '{': ++m; break;
			case '}': --m; break;
		}
		if (m == 0) break;
	}
	if (m !== 0) throw new Error('Cant make {} match at url:' + url);
	let obj = {};
	vm.runInNewContext('qq=' + js, obj);
	if (!obj.qq)
	{
		throw new Error('eval error, no g_qq, url:' + url);
	}
	obj.qq._url = url;
	obj.qq._html = text;
	let savetext = JSON.stringify(obj.qq);

	fs.writeFileSync(`${savefolder}\\Q-${obj.qq.id}.json`, savetext, { encoding: 'utf8' });
	meta.done[url] = true;
	++g_Count;
	if (g_Count % 14 == 0)
	{
		await wait(5000 + Math.floor(Math.random() * 2000));
	}
	else if (g_Count % 50 === 0)
	{
		console.log('long wait');
		await wait(60 * 1000);
	}
	else
	{
		await wait(200 + Math.floor(Math.random() * 50));
	}

	return js;*/
}
async function main()
{
//	for (let i = 1; i <= 53; ++i)
	{
		let url = `https://sh.lianjia.com/xiaoqu/`
		console.log(url)
		await grabpage(url);
	}
}



main().catch(console.error);


